#!/usr/bin/env python3
"""
大众点评爬虫 P4 版本 - 智能分页模块
实现真正的智能分页检测，替换固定页数限制
"""

import logging
import time
import re
from typing import Dict, List, Optional, Tuple, Any
from playwright.sync_api import Page
from dataclasses import dataclass

logger = logging.getLogger(__name__)

@dataclass
class PaginationInfo:
    """分页信息数据类"""
    current_page: int
    total_pages: Optional[int]
    has_next: bool
    has_prev: bool
    next_url: Optional[str]
    prev_url: Optional[str]
    page_info_text: str

class SmartPagination:
    """智能分页检测器 - 真正的智能分页检测"""
    
    def __init__(self):
        # 下一页按钮的常见选择器
        self.next_page_selectors = [
            'a:has-text("下一页"):not(.disabled)',
            'a[title*="下一页"]:not(.disabled)',
            'a.next:not(.disabled)',
            'a:has-text(">")',
            '.pagination .next:not(.disabled)',
            '.page-next:not(.disabled)',
            'a[href*="p"]:has-text("下一页")',
            '.pager .next:not(.disabled)'
        ]

        # 分页信息的常见选择器
        self.pagination_info_selectors = [
            '.pagination-info',
            '.page-info',
            '.pager-info',
            '.total-page',
            '.page-count'
        ]

        # 当前页码的选择器
        self.current_page_selectors = [
            '.pagination .current',
            '.page-current',
            '.active',
            '.selected'
        ]

        # 统计信息
        self.stats = {
            'pages_detected': 0,
            'smart_stops': 0,
            'fallback_stops': 0,
            'detection_errors': 0,
            'total_pages_detected': 0,
            'successful_detections': 0
        }

        # 固定抓取配置
        self.fallback_max_pages = 15
    
    def detect_pagination_info(self, page: Page) -> PaginationInfo:
        """检测当前页面的分页信息"""
        try:
            # 检测当前页码
            current_page = self._detect_current_page(page)

            # 检测总页数
            total_pages = self._detect_total_pages(page)

            # 检测下一页按钮
            has_next, next_url = self._detect_next_page(page)

            # 检测上一页按钮
            has_prev, prev_url = self._detect_prev_page(page)

            # 获取分页信息文本
            page_info_text = self._get_pagination_text(page)

            pagination_info = PaginationInfo(
                current_page=current_page,
                total_pages=total_pages,
                has_next=has_next,
                has_prev=has_prev,
                next_url=next_url,
                prev_url=prev_url,
                page_info_text=page_info_text
            )

            # 更新统计信息
            self.stats['pages_detected'] += 1
            if total_pages:
                self.stats['total_pages_detected'] = total_pages
                self.stats['successful_detections'] += 1

            logger.info(f"[SMART_PAGINATION] 分页检测: 当前页 {current_page}, 总页数 {total_pages}, 有下一页 {has_next}")

            return pagination_info

        except Exception as e:
            logger.error(f"[SMART_PAGINATION] 检测分页信息失败: {e}")
            self.stats['detection_errors'] += 1

            # 返回默认信息
            return PaginationInfo(
                current_page=1,
                total_pages=None,
                has_next=False,
                has_prev=False,
                next_url=None,
                prev_url=None,
                page_info_text=""
            )
    
    def _detect_current_page(self, page: Page) -> int:
        """检测当前页码"""
        try:
            for selector in self.current_page_selectors:
                try:
                    element = page.query_selector(selector)
                    if element:
                        text = element.inner_text().strip()
                        # 提取数字
                        match = re.search(r'\d+', text)
                        if match:
                            return int(match.group())
                except:
                    continue
            
            # 从URL中提取页码
            url = page.url
            match = re.search(r'p(\d+)', url)
            if match:
                return int(match.group(1))
            
            # 默认为第1页
            return 1
            
        except Exception as e:
            logger.debug(f"[SMART_PAGINATION] 检测当前页码失败: {e}")
            return 1
    
    def _detect_total_pages(self, page: Page) -> Optional[int]:
        """检测总页数"""
        try:
            # 尝试从分页信息中提取总页数
            for selector in self.pagination_info_selectors:
                try:
                    element = page.query_selector(selector)
                    if element:
                        text = element.inner_text()
                        # 匹配 "共X页" 或 "总共X页" 等模式
                        match = re.search(r'共\s*(\d+)\s*页|总共\s*(\d+)\s*页|共(\d+)页', text)
                        if match:
                            return int(match.group(1) or match.group(2) or match.group(3))
                except:
                    continue

            # 大众点评特定的分页检测
            max_page = self._detect_dianping_total_pages(page)
            if max_page:
                return max_page

            # 尝试从分页链接中找到最大页码
            page_links = page.query_selector_all('a[href*="p"]')
            max_page = 0

            for link in page_links:
                try:
                    href = link.get_attribute('href')
                    text = link.inner_text().strip()

                    # 检查是否是数字页码链接
                    if text.isdigit():
                        page_num = int(text)
                        max_page = max(max_page, page_num)

                    # 从URL中提取页码
                    if href:
                        match = re.search(r'p(\d+)', href)
                        if match:
                            page_num = int(match.group(1))
                            max_page = max(max_page, page_num)
                except:
                    continue

            return max_page if max_page > 0 else None

        except Exception as e:
            logger.debug(f"[SMART_PAGINATION] 检测总页数失败: {e}")
            return None

    def _detect_dianping_total_pages(self, page: Page) -> Optional[int]:
        """大众点评特定的总页数检测"""
        try:
            # 等待页面加载完成
            page.wait_for_load_state('networkidle', timeout=10000)

            # 使用更精确的分页容器选择器
            pagination_selectors = [
                '.page',  # 大众点评的分页容器
                'div:has(a:has-text("下一页"))',
                '.pagination',
                '.pager'
            ]

            for pagination_selector in pagination_selectors:
                try:
                    pagination_area = page.query_selector(pagination_selector)
                    if pagination_area:
                        logger.debug(f"[SMART_PAGINATION] 找到分页容器: {pagination_selector}")

                        # 查找所有链接
                        all_links = pagination_area.query_selector_all('a')
                        max_page = 0
                        found_pages = []

                        for link in all_links:
                            text = link.inner_text().strip()
                            href = link.get_attribute('href') or ''

                            # 检查文本是否为数字
                            if text.isdigit():
                                page_num = int(text)
                                max_page = max(max_page, page_num)
                                found_pages.append(page_num)

                            # 从URL中提取页码作为备用方法
                            elif 'p' in href:
                                match = re.search(r'p(\d+)', href)
                                if match:
                                    page_num = int(match.group(1))
                                    max_page = max(max_page, page_num)
                                    found_pages.append(page_num)

                        if max_page > 0:
                            logger.info(f"[SMART_PAGINATION] 大众点评检测到总页数: {max_page} (找到页码: {sorted(set(found_pages))}) (选择器: {pagination_selector})")
                            return max_page
                        else:
                            logger.debug(f"[SMART_PAGINATION] 选择器 {pagination_selector} 未找到有效页码")

                except Exception as e:
                    logger.debug(f"[SMART_PAGINATION] 选择器 {pagination_selector} 检测失败: {e}")
                    continue

            # 如果上述方法都失败，尝试更通用的方法
            logger.debug("[SMART_PAGINATION] 尝试通用页码检测方法")
            all_page_links = page.query_selector_all('a[href*="p"]')
            max_page = 0

            for link in all_page_links:
                try:
                    text = link.inner_text().strip()
                    href = link.get_attribute('href') or ''

                    if text.isdigit() and int(text) <= 1000:  # 合理的页码范围
                        page_num = int(text)
                        max_page = max(max_page, page_num)

                except:
                    continue

            if max_page > 0:
                logger.info(f"[SMART_PAGINATION] 通用方法检测到总页数: {max_page}")
                return max_page

            return None

        except Exception as e:
            logger.debug(f"[SMART_PAGINATION] 大众点评页数检测失败: {e}")
            return None
    
    def _detect_next_page(self, page: Page) -> Tuple[bool, Optional[str]]:
        """检测下一页按钮和链接"""
        try:
            # 大众点评特定的下一页检测
            next_page_selectors = [
                'a:has-text("下一页")',
                'a[title*="下一页"]',
                'a.next:not(.disabled)',
                '.pagination .next:not(.disabled)',
                '.page-next:not(.disabled)',
                'a[href*="p"]:has-text("下一页")',
                '.pager .next:not(.disabled)'
            ]

            for selector in next_page_selectors:
                try:
                    element = page.query_selector(selector)
                    if element and element.is_visible():
                        # 检查是否被禁用
                        class_name = element.get_attribute('class') or ''
                        if 'disabled' in class_name.lower():
                            continue

                        # 检查文本内容
                        text = element.inner_text().strip()
                        if "下一页" not in text and ">" not in text:
                            continue

                        # 获取链接
                        href = element.get_attribute('href')
                        if href:
                            if href.startswith('/'):
                                href = f"https://www.dianping.com{href}"
                            return True, href
                        else:
                            # 可能是JavaScript链接
                            return True, None
                except:
                    continue

            return False, None

        except Exception as e:
            logger.debug(f"[SMART_PAGINATION] 检测下一页失败: {e}")
            return False, None
    
    def _detect_prev_page(self, page: Page) -> Tuple[bool, Optional[str]]:
        """检测上一页按钮和链接"""
        try:
            prev_selectors = [
                'a.prev:not(.disabled)',
                'a[title*="上一页"]:not(.disabled)',
                'a:has-text("上一页"):not(.disabled)',
                'a:has-text("<")',
                '.pagination .prev:not(.disabled)'
            ]
            
            for selector in prev_selectors:
                try:
                    element = page.query_selector(selector)
                    if element and element.is_visible():
                        href = element.get_attribute('href')
                        if href:
                            if href.startswith('/'):
                                href = f"https://www.dianping.com{href}"
                            return True, href
                        else:
                            return True, None
                except:
                    continue
            
            return False, None
            
        except Exception as e:
            logger.debug(f"[SMART_PAGINATION] 检测上一页失败: {e}")
            return False, None
    
    def _get_pagination_text(self, page: Page) -> str:
        """获取分页信息文本"""
        try:
            for selector in self.pagination_info_selectors:
                try:
                    element = page.query_selector(selector)
                    if element:
                        return element.inner_text().strip()
                except:
                    continue
            
            # 尝试获取页面底部的分页信息
            pagination_area = page.query_selector('.pagination, .pager, .page-nav')
            if pagination_area:
                return pagination_area.inner_text().strip()
            
            return ""
            
        except Exception as e:
            logger.debug(f"[SMART_PAGINATION] 获取分页文本失败: {e}")
            return ""
    
    def has_next_page(self, page: Page) -> bool:
        """检查是否有下一页（主要接口）"""
        pagination_info = self.detect_pagination_info(page)
        return pagination_info.has_next
    
    def click_next_page(self, page: Page) -> bool:
        """点击下一页按钮"""
        try:
            for selector in self.next_page_selectors:
                try:
                    element = page.query_selector(selector)
                    if element and element.is_visible():
                        # 检查是否被禁用
                        class_name = element.get_attribute('class') or ''
                        if 'disabled' in class_name.lower():
                            continue
                        
                        # 点击下一页
                        element.click()
                        
                        # 等待页面加载
                        page.wait_for_load_state('networkidle', timeout=15000)
                        
                        logger.info("[SMART_PAGINATION] ✅ 成功点击下一页")
                        return True
                        
                except Exception as e:
                    logger.debug(f"[SMART_PAGINATION] 点击选择器 {selector} 失败: {e}")
                    continue
            
            logger.warning("[SMART_PAGINATION] ❌ 未找到可点击的下一页按钮")
            return False
            
        except Exception as e:
            logger.error(f"[SMART_PAGINATION] 点击下一页失败: {e}")
            return False
    
    def get_total_pages_estimate(self, page: Page, base_url: str, max_test_pages: int = 10) -> int:
        """估算总页数（用于测试类）"""
        logger.info(f"[SMART_PAGINATION] 开始估算总页数，最多测试 {max_test_pages} 页")
        
        try:
            # 首先尝试直接检测
            pagination_info = self.detect_pagination_info(page)
            if pagination_info.total_pages:
                logger.info(f"[SMART_PAGINATION] 直接检测到总页数: {pagination_info.total_pages}")
                return pagination_info.total_pages
            
            # 二分查找法估算总页数
            return self._binary_search_total_pages(base_url, max_test_pages)
            
        except Exception as e:
            logger.error(f"[SMART_PAGINATION] 估算总页数失败: {e}")
            return 1
    
    def _binary_search_total_pages(self, base_url: str, max_pages: int) -> int:
        """使用二分查找估算总页数"""
        from playwright.sync_api import sync_playwright
        from config.settings import get_cookie_string
        
        try:
            with sync_playwright() as p:
                browser = p.chromium.launch(headless=True)
                context = browser.new_context()
                
                # 添加Cookie
                cookies = []
                cookie_string = get_cookie_string()
                for cookie_pair in cookie_string.split('; '):
                    if '=' in cookie_pair:
                        name, value = cookie_pair.split('=', 1)
                        cookies.append({
                            'name': name.strip(),
                            'value': value.strip(),
                            'domain': '.dianping.com',
                            'path': '/'
                        })
                context.add_cookies(cookies)
                
                page = context.new_page()
                
                left, right = 1, max_pages
                last_valid_page = 1
                
                while left <= right:
                    mid = (left + right) // 2
                    test_url = f"{base_url}p{mid}" if mid > 1 else base_url
                    
                    try:
                        page.goto(test_url, timeout=15000)
                        page.wait_for_load_state('networkidle', timeout=10000)
                        
                        # 检查页面是否有效（有商铺数据）
                        shops = page.query_selector_all('li[class=""]')  # 商铺列表
                        
                        if len(shops) > 0:
                            last_valid_page = mid
                            left = mid + 1
                        else:
                            right = mid - 1
                            
                        time.sleep(1)  # 避免请求过快
                        
                    except Exception as e:
                        logger.debug(f"[SMART_PAGINATION] 测试页面 {mid} 失败: {e}")
                        right = mid - 1
                
                browser.close()
                
                logger.info(f"[SMART_PAGINATION] 二分查找估算总页数: {last_valid_page}")
                return last_valid_page
                
        except Exception as e:
            logger.error(f"[SMART_PAGINATION] 二分查找失败: {e}")
            return 1
    
    def create_pagination_test_report(self, city_name: str, category_name: str, 
                                    base_url: str) -> Dict[str, Any]:
        """创建分页测试报告"""
        logger.info(f"[SMART_PAGINATION] 创建分页测试报告: {city_name}-{category_name}")
        
        try:
            from playwright.sync_api import sync_playwright
            from config.settings import get_cookie_string
            
            with sync_playwright() as p:
                browser = p.chromium.launch(headless=True)
                context = browser.new_context()
                
                # 添加Cookie
                cookies = []
                cookie_string = get_cookie_string()
                for cookie_pair in cookie_string.split('; '):
                    if '=' in cookie_pair:
                        name, value = cookie_pair.split('=', 1)
                        cookies.append({
                            'name': name.strip(),
                            'value': value.strip(),
                            'domain': '.dianping.com',
                            'path': '/'
                        })
                context.add_cookies(cookies)
                
                page = context.new_page()
                page.goto(base_url, timeout=30000)
                page.wait_for_load_state('networkidle', timeout=20000)
                
                # 检测分页信息
                pagination_info = self.detect_pagination_info(page)
                
                # 估算总页数
                estimated_pages = self.get_total_pages_estimate(page, base_url)
                
                browser.close()
                
                report = {
                    'city_name': city_name,
                    'category_name': category_name,
                    'base_url': base_url,
                    'current_page': pagination_info.current_page,
                    'detected_total_pages': pagination_info.total_pages,
                    'estimated_total_pages': estimated_pages,
                    'has_next_page': pagination_info.has_next,
                    'pagination_text': pagination_info.page_info_text,
                    'test_time': time.time()
                }
                
                logger.info(f"[SMART_PAGINATION] 测试报告完成: 估算 {estimated_pages} 页")
                return report
                
        except Exception as e:
            logger.error(f"[SMART_PAGINATION] 创建测试报告失败: {e}")
            return {
                'error': str(e),
                'city_name': city_name,
                'category_name': category_name
            }
    
    def get_pagination_stats(self) -> Dict[str, Any]:
        """获取分页统计信息"""
        return {
            'pages_detected': self.stats['pages_detected'],
            'smart_stops': self.stats['smart_stops'],
            'fallback_stops': self.stats['fallback_stops'],
            'detection_errors': self.stats['detection_errors'],
            'total_pages_detected': self.stats.get('total_pages_detected', 0),
            'successful_detections': self.stats.get('successful_detections', 0),
            'success_rate': round(
                (self.stats['pages_detected'] - self.stats['detection_errors']) /
                max(self.stats['pages_detected'], 1) * 100, 2
            )
        }

    def should_use_fallback(self, pagination_info: PaginationInfo) -> bool:
        """判断是否应该使用后备方法"""
        # 如果无法检测到总页数或下一页按钮，使用后备方法
        if not pagination_info.total_pages and not pagination_info.has_next:
            logger.warning("[SMART_PAGINATION] 智能分页失败，将使用固定抓取方法")
            self.stats['fallback_stops'] += 1
            return True
        return False

    def get_max_pages_for_crawling(self, pagination_info: PaginationInfo) -> int:
        """获取爬取的最大页数"""
        if self.should_use_fallback(pagination_info):
            logger.info(f"[SMART_PAGINATION] 使用后备方法，固定抓取 {self.fallback_max_pages} 页")
            return self.fallback_max_pages

        if pagination_info.total_pages:
            logger.info(f"[SMART_PAGINATION] 智能检测到总页数: {pagination_info.total_pages}")
            return pagination_info.total_pages

        # 如果有下一页但没有总页数，使用较大的默认值
        logger.info(f"[SMART_PAGINATION] 有下一页但无总页数，使用默认最大页数: {self.fallback_max_pages}")
        return self.fallback_max_pages

    def is_last_page(self, page: Page, current_page: int, max_pages: int) -> bool:
        """判断是否为最后一页"""
        try:
            # 检查是否有下一页按钮
            pagination_info = self.detect_pagination_info(page)

            # 如果没有下一页按钮，说明是最后一页
            if not pagination_info.has_next:
                logger.info(f"[SMART_PAGINATION] 检测到最后一页: 第{current_page}页（无下一页按钮）")
                self.stats['smart_stops'] += 1
                return True

            # 如果当前页数达到最大页数
            if current_page >= max_pages:
                logger.info(f"[SMART_PAGINATION] 达到最大页数: 第{current_page}页")
                return True

            return False

        except Exception as e:
            logger.error(f"[SMART_PAGINATION] 判断最后一页失败: {e}")
            return current_page >= max_pages
